package source;

import java.util.*;
import java.io.*;

public class Classification {
	public static void main(String args[]) {
		BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
		String filename;
		String path = "C:\\Users\\Sony\\Desktop\\web";
		int totalDocs = 0;
		Indexer d = new Indexer();
		d.setTotalClasses(6);
		try {
			d.addStopWords();// tested
			totalDocs += d.getDocuments(path);// tested
			d.generateChi();
			//d.displayPostingsHash();
			System.out.println("TotalDocs : " + totalDocs);
			while (true) {
				filename = br.readLine();
				// "C:\\Users\\VAIO\\Desktop\\49960"
				d.assignClass(filename, totalDocs);
			}
		} catch (Exception ex) {
			System.out.println("Exception occured : " + ex.getMessage());
			ex.printStackTrace();
		}
	}
}

class Indexer {

	HashSet<String> stopwords = new HashSet<String>();
	FileInputStream fin;
	DataInputStream din;
	BufferedReader br;
	HashMap<String, Integer> docId;
	ArrayList<String> className = new ArrayList<String>();
	LinkedHashMap<String, int[]> classTermIndex = new LinkedHashMap<String, int[]>();
	HashMap<String, HashMap<Integer, Integer>> posting = new HashMap<String, HashMap<Integer, Integer>>();
	HashMap<Integer, Integer> docCount = new HashMap<Integer, Integer>();
	HashMap<String, int[]> docTermCount = new HashMap<String, int[]>();
	HashMap<String, double[]> chiValue = new HashMap<String, double[]>();
	double  avg = 0;

	int totalNumberOfDocuments = 0;
	int totalClasses;
	int[] noOfDocInClass;
	int[] evidenceProbability;
	int[][] docClassMapping = new int[20][50000];
	int[][] wordDocCount = new int[50000][20];
	int wordCount;

	public void setTotalClasses(int totalClasses) {
		this.totalClasses = totalClasses;
		this.noOfDocInClass = new int[totalClasses];
		this.evidenceProbability = new int[totalClasses];
	}

	int getDocuments(String path) throws IOException {
		docId = new HashMap<String, Integer>();
		// textFiles = new ArrayList<String>();
		int i = 0;
		int classId = 0;
		String filepath;
		// Iterator iter = stopwords.iterator();
		File dir = new File(path);
		for (File file : dir.listFiles()) {
			className.add(file.getName());
			System.out.println("Class : " + file.getName());
			for (File f1 : file.listFiles()) {
				docClassMapping[classId][i] = 1;
				// System.out.println(classId+" "+i+" "+docClassMapping[classId][i]);
				docId.put(f1.getName(), i++);
				filepath = path + "\\" + file.getName() + "\\" + f1.getName();
				removeStopWords(filepath, i, classId);
				noOfDocInClass[classId]++;
				totalNumberOfDocuments++;
			}
			classId++;
		}
		System.out.println("successfully got the documents");
		return totalNumberOfDocuments;

	}

	void generateChi() {
		int a = 0, b, c, d, totalDocsHavingTerm = 0;
		int i;
		int t[];
		for (Map.Entry<String, int[]> me : docTermCount.entrySet()) {
			// System.out.print("\n" + me.getKey() + "-->");
			double[] temp = new double[totalClasses];
			t = docTermCount.get(me.getKey());
			totalDocsHavingTerm = 0;
			for (i = 0; i < totalClasses; i++) {
				totalDocsHavingTerm += t[i];
			}
			for (i = 0; i < totalClasses; i++) {
				a = t[i];
				b = totalDocsHavingTerm - a;
				c = noOfDocInClass[i] - a;
				d = totalNumberOfDocuments - totalDocsHavingTerm - c;

				temp[i] = (totalNumberOfDocuments * (Math.pow((a * d - c * b),
						2))) / ((a + c) * (b + d) * (a + b) * (c + d));
				//				// chiValue.put((me.getKey(),)
				// System.out.print(temp[i] + " ");

				//				temp[i] = Math.log((a*totalNumberOfDocuments)/((a+c)*(a+b)));

				//				if(me.getKey().equals("ending"))
				//				 {
				//				 System.out.println("a: "+a+", b: "+b+", c:"+c+", d:"+d);
				//				 System.out.println(temp[i]);
				//				 }

			}
						for (i = 0; i < 2; i++)
							System.out.println("--" + temp[i]);

			chiValue.put(me.getKey(), temp);

		}

		double temp;
		double[] chiValueTemp ;
		double[] classProbability = new double[totalClasses];

		for (i = 0; i < totalClasses; i++) {
			classProbability[i] = noOfDocInClass[i] / (double)totalNumberOfDocuments;
			System.out.println("Probab : " +i+"  "+classProbability[i]); 
		}

		for (Map.Entry<String, double[]> me : chiValue.entrySet()) {
			temp = 0;
			chiValueTemp = me.getValue();
			for(i=0;i<totalClasses;i++){
				if(!Double.isNaN(chiValueTemp[i])&&!Double.isInfinite(chiValueTemp[i]))
					temp += classProbability[i]*chiValueTemp[i];
			}
			avg+= temp;
		}

		System.out.println("Words : "+chiValue.size());

		avg /= chiValue.size();

		System.out.println("Average : "+avg);

		// chi=totalNumberOfDocuments;
	}

	void removeStopWords(String path, int docId, int classId)
			throws IOException {
		String word;
		int[] tempClassTermCount;
		Scanner input = new Scanner(new File(path));
		// System.out.println("indexing document no. : " + docId + path);
		while (input.hasNext()) {
			word = input.next();
			word = word.toLowerCase();
			if (!stopwords.contains(word)) {
				word = word.replaceAll("[^A-Za-z]", "");
				if (classTermIndex.containsKey(word))
					tempClassTermCount = classTermIndex.get(word);
				else
					tempClassTermCount = new int[totalClasses];

				tempClassTermCount[classId]++;
				classTermIndex.put(word, tempClassTermCount);

				// char arr[] = word.toCharArray();
				 //Stemmer s = new Stemmer();
				try {
					
					  //s.add(arr, word.length()); s.stem(); word = s.toString();
					 
					word = word.replaceAll("[^A-Za-z]", "");
					// System.out.println("found word : " + word);
					if (!posting.containsKey(word)) {
						if (word.length() > 3 && word.length() < 15) {
							// System.out.println("CAME HERE");
							docCount = new HashMap<Integer, Integer>();
							docCount.put(docId, 1);
							posting.put(word, docCount);
							int a[] = new int[totalClasses];
							a[classId]++;
							docTermCount.put(word, a);
							// System.out.println("word inserted : " +
							// word);
						}
					} else {
						HashMap<Integer, Integer> strhm = new HashMap<Integer, Integer>();
						strhm = posting.get(word);
						if (strhm.get(docId) == null) {// if the already
							// existing word in the
							// postings list, has
							// occured in a document
							// the first time
							strhm.put(docId, 1);
							int a[] = docTermCount.get(word);
							a[classId]++;
							docTermCount.put(word, a);
						} else {
							strhm.put(docId, (strhm.get(docId)) + 1);
						}
						posting.put(word, strhm);
					}

				} catch (Exception e) {
					System.out.println("exception occured");

				}
			}
		}
	}

	void addStopWords() throws IOException {
		stopwords = new HashSet<String>();
		fin = new FileInputStream("C:\\Users\\Sony\\Desktop\\stop.txt");
		din = new DataInputStream(fin);
		br = new BufferedReader(new InputStreamReader(din));
		String line;
		while ((line = br.readLine()) != null) {
			// System.out.println(line);
			stopwords.add(line);
		}
		System.out.println("successfully added the stop words");
	}

	void displayPostingsHash() {
		/*
		 * for (Map.Entry<String, HashMap<Integer, Integer>> me : posting
		 * .entrySet()) { docCount = me.getValue(); System.out.print(me.getKey()
		 * + "-->"); for (Map.Entry<Integer, Integer> me1 : docCount.entrySet())
		 * { System.out.print("(" + me1.getKey() + "," + me1.getValue() + ")");
		 * } System.out.println(""); }
		 */
		System.out.println("words :" + docTermCount.size());
		int i;
		double[] temp;
		for (Map.Entry<String, double[]> me : chiValue.entrySet()) {
			System.out.print("\n" + me.getKey() + "-->");
			temp = me.getValue();
			for (i = 0; i < totalClasses; i++)
				System.out.print(temp[i] + " ");

		}

		// for(i=0;i<totalClasses;i++){
		// System.out.println(i+" "+noOfDocInClass[i]);
		// // }

		// for (Map.Entry<String, int[]> me : docTermCount.entrySet()) {
		//
		//
		// }
		System.out.println("Total Docs : " + totalNumberOfDocuments);

	}

	float assignClass(String fileName, int totalDocs) {
		// TODO: create index for this doc first
		int i;
		double[] probability = new double[totalClasses];
		HashMap<String, Integer> docWordCount = new HashMap<String, Integer>();
		String word;

		// generate prior evidence
		System.out.println("Prior Probability : ");
		for (i = 0; i < totalClasses; i++) {
			probability[i] = Math.log((float) noOfDocInClass[i] / totalDocs);
			System.out.println("Probab : " + noOfDocInClass[i] + " "
					+ probability[i]);
		}

		// generate index for incoming file
		try {
			Scanner input = new Scanner(new File(fileName));
			while (input.hasNext()) {
				word = input.next();
				word = word.toLowerCase();
				if (!stopwords.contains(word) && word.length() > 0) {
					word = word.replaceAll("[^A-Za-z]", "");
					if (docWordCount.containsKey(word))
						docWordCount.put(word, docWordCount.get(word) + 1);
					else
						docWordCount.put(word, 1);
				}
			}
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		// generate the denominator
		double denominator[] = new double[totalClasses];
		int temp[];
		double tempChiValue[];
		for (Map.Entry<String, int[]> entry : classTermIndex.entrySet()) {
			temp = entry.getValue();

			tempChiValue = chiValue.get(entry.getKey());
			if(tempChiValue!=null)
				for (i = 0; i < totalClasses; i++) {
					if(tempChiValue[i]>avg)
						denominator[i] += (temp[i] + 1);
					else
						denominator[i] += 1;
				}
		}

		for (Map.Entry<String, Integer> entry : docWordCount.entrySet()) {
			tempChiValue = chiValue.get(entry.getKey());
			if(tempChiValue!=null){
				temp = classTermIndex.get(entry.getKey());
				if (temp == null&&tempChiValue[i]<avg) {
					for (i = 0; i < totalClasses; i++) {
							probability[i] += Math.log(1 / denominator[i]);
					}
				} else {
					for (i = 0; i < totalClasses; i++) {
							probability[i] += Math.log(temp[i] + 1 / denominator[i]);
					}
				}
			}
		}

		System.out.println("Final Probability : ");

		for (i = 0; i < totalClasses; i++) {
			System.out.println(noOfDocInClass[i] + " " + probability[i]
					+ ": for " + className.get(i));
		}

		int max = 0;
		for (i = 1; i < totalClasses; i++) {
			if (probability[max] < probability[i])
				max = i;
		}

		System.out.println("The Document belong to the class : "
				+ className.get(max));

		// System.out.println("Term "+word+" occured "+termOcc+" times.");

		return 0;

	}
}